var citiesList = "beijing";

//var cities = citiesList.split(",");
var cities = []
process.argv.forEach(function (val, index, array) {
    if (index >= 2)
        cities.push(val)
});

var path = require('path'), request = require('request'), cheerio = require('cheerio');
var Crawler = require('../../lib/crawler');
//var connection = 'mongodb://localhost:27017/66xiaoqu';
//var connection = 'mongodb://192.168.1.123:27017/66xiaoqu';
var connection = 'mongodb://120.55.126.201:27017/Communities';
var async = require('async');

var MongoClient = require('mongodb').MongoClient, assert = require('assert');

var dbconn = null;

cities.forEach(function(city){
//async.eachSeries(cities, function (city) {

    var url = 'http://' + city + '.anjuke.com/community';
    var dbCollection = 'xq_anjuke_' + city;
    var totalPages = 1;
    var numPerPage = 10;
    var totalComms = 0;
    var counter = 0;
    var db = null;

    function getTotalPages(callback) {
        // Parse Total Pages
        request(url, function (err, resp, body) {
            if (!err && resp.statusCode == 200) {
                var $ = cheerio.load(body);
                var totalComm = $("span.spResult", "div.listPager").text().replace(/[^0-9\.]+/g, '');
                totalComms = totalComm
                var allPages = Math.floor(parseInt(totalComm) / numPerPage) + 1;
                totalPages = allPages;
                console.log("Total Pages for " + city.toUpperCase() + ": " + allPages);
            }
            callback();
        });
    };

    function run() {

        var manifest = {
            target: {
                url: url + '/W0QQpZ{{number}}',
                ranges: {
                    number: [1, totalPages, 1]
                },
                interval: 100 // Wait 0.5 seconds between each access
            },
            plugins: {
                mongodb: {
                    connection: connection
                },
                csv: true
            },

            extractors: [{
                name: '66xiaoqu',
                root: '//ul[@class="list"]/li[@class="list_item"]',
                fields: {
                    name: '//div[@class="details"]/div[@class="t_b"]/a[@class="t"]/text()',
                    href: '//div[@class="details"]/div[@class="t_b"]/a[@class="t"]/@href',
                    addr: '//div[@class="details"]/p[1]/text()',
                    fdate: '//div[@class="details"]/p[2]/text()',
                    map: '//div[@class="details"]/p[@class="p_links"]/a[1]/@href',
                    price: '//span[@class="price"]/span["sp1"]/text()',
                }
            }
            ]
        };

        function insertDocuments(doc) {

            var collection = dbconn.collection(dbCollection)

            var regex = /^http:(.*)#l1=(.*)&l2=(.*)&l3=(.*)&(.*)&commname=(.*)&commid=(.*)(&?)/;
            var result = doc.map.match(regex);
            var lat = result[2];
            var lng = result[3];
            var commname = result[6];
            var commid = parseInt(result[7]);

            var new_doc = {
                name: doc.name,
                address: doc.addr.replace('&amp;nbsp;', ' '),
                dateBuilt: doc.fdate.trim().replace('竣工日期：', ''),
                latitude: parseFloat(lat),
                longitude: parseFloat(lng),
                commName: commname,
                commId: commid,
                price: parseInt(doc.price),
                href: doc.href,
                mapURL: doc.map,
                source:'anjuke'
            }

            collection.update({"commId":commid},new_doc, {"upsert":true},function (err, result) {
                    counter ++
                    //assert.equal(err, null);
                    //assert.equal(1, result.result.n);
                    //assert.equal(1, result.ops.length);
                    console.log("Inserted %s into %s (%s of %s)", doc.name, dbCollection, counter, totalComms);
                    //callback(result);
                });
        };

        var job = new Crawler(manifest);

        job.on('data', function (data) {
            //console.log(job._manifest)
            data.data.forEach(
                function (doc) {
                    insertDocuments(doc);
                })
        });

        job.on('error', function (err) {
            throw error;
        });

        job.on('end', function () {
            console.log('The job is done');
            db.close()
            // not exactly,
        });

        function init(callback) {
            // Connect to MongoDB
            MongoClient.connect(connection,
                function (err, db) {
                    assert.equal(null, err);
                    console.log("Connected correctly to server");
                    dbconn = db;
                    //collection = dbconn.collection(dbCollection)
                    callback(null, 'init');
                });
        }

        function start(callback) {
            console.log("starting...");
            job.start();
            callback(null, 'start');
        }

        function finish(callback) {
            console.log("closing db...");
//    dbconn.close();
            callback(null, 'end');
        }


// Start our job
        async.series([
                //getTotalPages,
                init,
                start,
                finish
            ]
            , function (err, results) {
                console.log(results);
                console.log('END');
            });
    }

    // Main method
    getTotalPages(run);
    //run()
});

/*
 node /workspace/CrawlerJS/examples/cities/anjuke-master.js

 shanghai
 beijing guangzhou shenzhen chongqing
 hangzhou suzhou chengdu changsha nanjing

 tianjin
 */
